From 047fb79307dd9b3212669ec617edbab7b0b3dc30 Mon Sep 17 00:00:00 2001 From: "kaf24@scramble.cl.cam.ac.uk" Date: Wed, 2 Mar 2005 17:53:54 +0000 Subject: [PATCH] bitkeeper revision 1.1236.1.46 (4225fdb2Fsz5LfXWKyVON4u_POTHkQ) Credit-based rate limiting in net backend. From Ross Mcilroy. Signed-off-by: Keir Fraser --- .../drivers/xen/netback/common.h | 1 + .../drivers/xen/netback/control.c | 9 +++- .../drivers/xen/netback/interface.c | 39 +++++++++++++-- .../drivers/xen/netback/netback.c | 49 ++++++++++--------- tools/python/xen/lowlevel/xu/xu.c | 13 +++++ tools/python/xen/xend/XendClient.py | 7 +++ tools/python/xen/xend/XendDomain.py | 9 ++++ tools/python/xen/xend/XendDomainInfo.py | 12 +++++ tools/python/xen/xend/server/SrvDomain.py | 9 ++++ tools/python/xen/xend/server/messages.py | 4 ++ tools/python/xen/xend/server/netif.py | 25 +++++++++- tools/python/xen/xm/main.py | 17 +++++++ xen/include/public/io/domain_controller.h | 17 +++++++ 13 files changed, 181 insertions(+), 30 deletions(-) diff --git a/linux-2.6.10-xen-sparse/drivers/xen/netback/common.h b/linux-2.6.10-xen-sparse/drivers/xen/netback/common.h index c5146c4860..dfb750ee36 100644 --- a/linux-2.6.10-xen-sparse/drivers/xen/netback/common.h +++ b/linux-2.6.10-xen-sparse/drivers/xen/netback/common.h @@ -78,6 +78,7 @@ typedef struct netif_st { void netif_create(netif_be_create_t *create); void netif_destroy(netif_be_destroy_t *destroy); +void netif_creditlimit(netif_be_creditlimit_t *creditlimit); void netif_connect(netif_be_connect_t *connect); int netif_disconnect(netif_be_disconnect_t *disconnect, u8 rsp_id); void netif_disconnect_complete(netif_t *netif); diff --git a/linux-2.6.10-xen-sparse/drivers/xen/netback/control.c b/linux-2.6.10-xen-sparse/drivers/xen/netback/control.c index 6ca0ba2c75..bac89b16c6 100644 --- a/linux-2.6.10-xen-sparse/drivers/xen/netback/control.c +++ b/linux-2.6.10-xen-sparse/drivers/xen/netback/control.c @@ -21,12 +21,17 @@ static void netif_ctrlif_rx(ctrl_msg_t *msg, unsigned long id) if ( msg->length != sizeof(netif_be_destroy_t) ) goto parse_error; netif_destroy((netif_be_destroy_t *)&msg->msg[0]); - break; + break; + case CMSG_NETIF_BE_CREDITLIMIT: + if ( msg->length != sizeof(netif_be_creditlimit_t) ) + goto parse_error; + netif_creditlimit((netif_be_creditlimit_t *)&msg->msg[0]); + break; case CMSG_NETIF_BE_CONNECT: if ( msg->length != sizeof(netif_be_connect_t) ) goto parse_error; netif_connect((netif_be_connect_t *)&msg->msg[0]); - break; + break; case CMSG_NETIF_BE_DISCONNECT: if ( msg->length != sizeof(netif_be_disconnect_t) ) goto parse_error; diff --git a/linux-2.6.10-xen-sparse/drivers/xen/netback/interface.c b/linux-2.6.10-xen-sparse/drivers/xen/netback/interface.c index 682ece07d9..6e084c4660 100644 --- a/linux-2.6.10-xen-sparse/drivers/xen/netback/interface.c +++ b/linux-2.6.10-xen-sparse/drivers/xen/netback/interface.c @@ -3,7 +3,7 @@ * * Network-device interface management. * - * Copyright (c) 2004, Keir Fraser + * Copyright (c) 2004-2005, Keir Fraser */ #include "common.h" @@ -140,7 +140,7 @@ void netif_create(netif_be_create_t *create) netif->credit_bytes = netif->remaining_credit = ~0UL; netif->credit_usec = 0UL; - /*init_ac_timer(&new_vif->credit_timeout);*/ + init_timer(&netif->credit_timeout); pnetif = &netif_hash[NETIF_HASH(domid, handle)]; while ( *pnetif != NULL ) @@ -234,6 +234,38 @@ void netif_destroy(netif_be_destroy_t *destroy) destroy->status = NETIF_BE_STATUS_OKAY; } +void netif_creditlimit(netif_be_creditlimit_t *creditlimit) +{ + domid_t domid = creditlimit->domid; + unsigned int handle = creditlimit->netif_handle; + netif_t *netif; + + netif = netif_find_by_handle(domid, handle); + if ( unlikely(netif == NULL) ) + { + DPRINTK("netif_creditlimit attempted for non-existent netif" + " (%u,%u)\n", creditlimit->domid, creditlimit->netif_handle); + creditlimit->status = NETIF_BE_STATUS_INTERFACE_NOT_FOUND; + return; + } + + /* Set the credit limit (reset remaining credit to new limit). */ + netif->credit_bytes = netif->remaining_credit = creditlimit->credit_bytes; + netif->credit_usec = creditlimit->period_usec; + + if ( netif->status == CONNECTED ) + { + /* + * Schedule work so that any packets waiting under previous credit + * limit are dealt with (acts like a replenishment point). + */ + netif->credit_timeout.expires = jiffies; + netif_schedule_work(netif); + } + + creditlimit->status = NETIF_BE_STATUS_OKAY; +} + void netif_connect(netif_be_connect_t *connect) { domid_t domid = connect->domid; @@ -245,9 +277,6 @@ void netif_connect(netif_be_connect_t *connect) pgprot_t prot; int error; netif_t *netif; -#if 0 - struct net_device *eth0_dev; -#endif netif = netif_find_by_handle(domid, handle); if ( unlikely(netif == NULL) ) diff --git a/linux-2.6.10-xen-sparse/drivers/xen/netback/netback.c b/linux-2.6.10-xen-sparse/drivers/xen/netback/netback.c index 71a3422129..cd20eda5bc 100644 --- a/linux-2.6.10-xen-sparse/drivers/xen/netback/netback.c +++ b/linux-2.6.10-xen-sparse/drivers/xen/netback/netback.c @@ -7,7 +7,7 @@ * reference front-end implementation can be found in: * drivers/xen/netfront/netfront.c * - * Copyright (c) 2002-2004, K A Fraser + * Copyright (c) 2002-2005, K A Fraser */ #include "common.h" @@ -380,14 +380,13 @@ void netif_deschedule_work(netif_t *netif) remove_from_net_schedule_list(netif); } -#if 0 + static void tx_credit_callback(unsigned long data) { netif_t *netif = (netif_t *)data; netif->remaining_credit = netif->credit_bytes; netif_schedule_work(netif); } -#endif static void net_tx_action(unsigned long unused) { @@ -471,42 +470,48 @@ static void net_tx_action(unsigned long unused) continue; } - netif->tx->req_cons = ++netif->tx_req_cons; - - /* - * 1. Ensure that we see the request when we copy it. - * 2. Ensure that frontend sees updated req_cons before we check - * for more work to schedule. - */ - mb(); - + rmb(); /* Ensure that we see the request before we copy it. */ memcpy(&txreq, &netif->tx->ring[MASK_NETIF_TX_IDX(i)].req, sizeof(txreq)); -#if 0 /* Credit-based scheduling. */ - if ( tx.size > netif->remaining_credit ) + if ( txreq.size > netif->remaining_credit ) { - s_time_t now = NOW(), next_credit = - netif->credit_timeout.expires + MICROSECS(netif->credit_usec); - if ( next_credit <= now ) + unsigned long now = jiffies; + unsigned long next_credit = + netif->credit_timeout.expires + + msecs_to_jiffies(netif->credit_usec / 1000); + + /* Timer could already be pending in some rare cases. */ + if ( timer_pending(&netif->credit_timeout) ) + break; + + /* Already passed the point at which we can replenish credit? */ + if ( time_after_eq(now, next_credit) ) { netif->credit_timeout.expires = now; netif->remaining_credit = netif->credit_bytes; } - else + + /* Still too big to send right now? Then set a timer callback. */ + if ( txreq.size > netif->remaining_credit ) { netif->remaining_credit = 0; netif->credit_timeout.expires = next_credit; netif->credit_timeout.data = (unsigned long)netif; netif->credit_timeout.function = tx_credit_callback; - netif->credit_timeout.cpu = smp_processor_id(); - add_ac_timer(&netif->credit_timeout); + add_timer_on(&netif->credit_timeout, smp_processor_id()); break; } } - netif->remaining_credit -= tx.size; -#endif + netif->remaining_credit -= txreq.size; + + /* + * Why the barrier? It ensures that the frontend sees updated req_cons + * before we check for more work to schedule. + */ + netif->tx->req_cons = ++netif->tx_req_cons; + mb(); netif_schedule_work(netif); diff --git a/tools/python/xen/lowlevel/xu/xu.c b/tools/python/xen/lowlevel/xu/xu.c index 5444bcd171..54f59dff04 100644 --- a/tools/python/xen/lowlevel/xu/xu.c +++ b/tools/python/xen/lowlevel/xu/xu.c @@ -698,6 +698,13 @@ static PyObject *xu_message_get_payload(PyObject *self, PyObject *args) C2P(netif_be_destroy_t, netif_handle, Int, Long); C2P(netif_be_destroy_t, status, Int, Long); return dict; + case TYPE(CMSG_NETIF_BE, CMSG_NETIF_BE_CREDITLIMIT): + C2P(netif_be_creditlimit_t, domid, Int, Long); + C2P(netif_be_creditlimit_t, netif_handle, Int, Long); + C2P(netif_be_creditlimit_t, credit_bytes, Int, Long); + C2P(netif_be_creditlimit_t, period_usec, Int, Long); + C2P(netif_be_creditlimit_t, status, Int, Long); + return dict; case TYPE(CMSG_NETIF_BE, CMSG_NETIF_BE_CONNECT): C2P(netif_be_connect_t, domid, Int, Long); C2P(netif_be_connect_t, netif_handle, Int, Long); @@ -916,6 +923,12 @@ static PyObject *xu_message_new(PyObject *self, PyObject *args) P2C(netif_be_destroy_t, domid, u32); P2C(netif_be_destroy_t, netif_handle, u32); break; + case TYPE(CMSG_NETIF_BE, CMSG_NETIF_BE_CREDITLIMIT): + P2C(netif_be_creditlimit_t, domid, u32); + P2C(netif_be_creditlimit_t, netif_handle, u32); + P2C(netif_be_creditlimit_t, credit_bytes, u32); + P2C(netif_be_creditlimit_t, period_usec, u32); + break; case TYPE(CMSG_NETIF_BE, CMSG_NETIF_BE_CONNECT): P2C(netif_be_connect_t, domid, u32); P2C(netif_be_connect_t, netif_handle, u32); diff --git a/tools/python/xen/xend/XendClient.py b/tools/python/xen/xend/XendClient.py index 4e733b7fde..cdd0fea929 100644 --- a/tools/python/xen/xend/XendClient.py +++ b/tools/python/xen/xend/XendClient.py @@ -278,6 +278,13 @@ class Xend: { 'op' : 'maxmem_set', 'memory' : memory }) + def xend_domain_vif_limit(self, id, vif, credit, period): + return self.xendPost(self.domainurl(id), + { 'op' : 'vif_credit_limit', + 'vif' : vif, + 'credit' : credit, + 'period' : period }) + def xend_domain_vifs(self, id): return self.xendGet(self.domainurl(id), { 'op' : 'vifs' }) diff --git a/tools/python/xen/xend/XendDomain.py b/tools/python/xen/xend/XendDomain.py index 1b01b43cfa..9089c75fb6 100644 --- a/tools/python/xen/xend/XendDomain.py +++ b/tools/python/xen/xend/XendDomain.py @@ -723,6 +723,15 @@ class XendDomain: dominfo = self.domain_lookup(id) return dominfo.get_device_by_index(type, idx) + def domain_vif_credit_limit(self, id, vif, credit, period): + """Limit the vif's transmission rate + """ + dominfo = self.domain_lookup(id) + try: + return dominfo.limit_vif(vif, credit, period) + except Exception, ex: + raise XendError(str(ex)) + def domain_vif_ls(self, id): """Get list of virtual network interface (vif) indexes for a domain. diff --git a/tools/python/xen/xend/XendDomainInfo.py b/tools/python/xen/xend/XendDomainInfo.py index b2a8e6caea..db2091ce08 100644 --- a/tools/python/xen/xend/XendDomainInfo.py +++ b/tools/python/xen/xend/XendDomainInfo.py @@ -614,6 +614,18 @@ class XendDomainInfo: def get_device_recreate(self, type, index): return self.get_device_savedinfo(type, index) or self.recreate + def limit_vif(self, vif, credit, period): + """Limit the rate of a virtual interface + @param vif: vif + @param credit: vif credit in bytes + @param period: vif period in uSec + @return: 0 on success + """ + + ctrl = xend.netif_create(self.dom, recreate=self.recreate) + d = ctrl.limitDevice(vif, credit, period) + return d + def add_config(self, val): """Add configuration data to a virtual machine. diff --git a/tools/python/xen/xend/server/SrvDomain.py b/tools/python/xen/xend/server/SrvDomain.py index d73a39bff5..2469a62894 100644 --- a/tools/python/xen/xend/server/SrvDomain.py +++ b/tools/python/xen/xend/server/SrvDomain.py @@ -164,6 +164,15 @@ class SrvDomain(SrvDir): d = fn(req.args, {'dom': self.dom.id}) return d + def op_vif_credit_limit(self, op, req): + fn = FormFn(self.xd.domain_vif_credit_limit, + [['dom', 'str'], + ['vif', 'int'], + ['credit', 'int'], + ['period', 'int']]) + val = fn(req.args, {'dom': self.dom.id}) + return val + def op_vifs(self, op, req): devs = self.xd.domain_vif_ls(self.dom.id) return [ dev.sxpr() for dev in devs ] diff --git a/tools/python/xen/xend/server/messages.py b/tools/python/xen/xend/server/messages.py index 4f9f9119a2..96e98416f7 100644 --- a/tools/python/xen/xend/server/messages.py +++ b/tools/python/xen/xend/server/messages.py @@ -150,6 +150,7 @@ CMSG_NETIF_BE_CREATE = 0 CMSG_NETIF_BE_DESTROY = 1 CMSG_NETIF_BE_CONNECT = 2 CMSG_NETIF_BE_DISCONNECT = 3 +CMSG_NETIF_BE_CREDITLIMIT = 4 CMSG_NETIF_BE_DRIVER_STATUS = 32 NETIF_INTERFACE_STATUS_CLOSED = 0 #/* Interface doesn't exist. */ @@ -173,6 +174,9 @@ netif_formats = { 'netif_be_destroy_t': (CMSG_NETIF_BE, CMSG_NETIF_BE_DESTROY), + 'netif_be_creditlimit_t': + (CMSG_NETIF_BE, CMSG_NETIF_BE_CREDITLIMIT), + 'netif_be_driver_status_t': (CMSG_NETIF_BE, CMSG_NETIF_BE_DRIVER_STATUS), diff --git a/tools/python/xen/xend/server/netif.py b/tools/python/xen/xend/server/netif.py index 00ad1f138b..2518a155ba 100755 --- a/tools/python/xen/xend/server/netif.py +++ b/tools/python/xen/xend/server/netif.py @@ -346,7 +346,21 @@ class NetDev(controller.SplitDev): vif = val['netif_handle'] self.status = NETIF_INTERFACE_STATUS_CONNECTED self.reportStatus() - + + def send_be_creditlimit(self, credit, period): + msg = packMsg('netif_be_creditlimit_t', + { 'domid' : self.controller.dom, + 'netif_handle' : self.vif, + 'credit_bytes' : credit, + 'period_usec' : period }) + d = defer.Deferred() + d.addCallback(self.respond_be_creditlimit) + self.getBackendInterface().writeRequest(msg, response=d) + + def respond_be_creditlimit(self, msg): + val = unpackMsg('netif_be_creditlimit_t', msg) + return self + def reportStatus(self, resp=0): msg = packMsg('netif_fe_interface_status_t', { 'handle' : self.vif, @@ -430,6 +444,15 @@ class NetifController(controller.SplitController): d = dev.attach() return d + def limitDevice(self, vif, credit, period): + if vif not in self.devices: + raise XendError('device does not exist for credit limit: vif' + + str(self.dom) + '.' + str(vif)) + + dev = self.devices[vif] + d = dev.send_be_creditlimit(credit, period) + return d + def recv_fe_driver_status(self, msg, req): if not req: return print diff --git a/tools/python/xen/xm/main.py b/tools/python/xen/xm/main.py index 411da44106..689d45bd86 100644 --- a/tools/python/xen/xm/main.py +++ b/tools/python/xen/xm/main.py @@ -717,6 +717,23 @@ class ProgLog(Prog): xm.prog(ProgLog) +class ProgVifCreditLimit(Prog): + group = 'vif' + name= "vif-limit" + info = """Limit the transmission rate of a virtual network interface.""" + + def help(self, args): + print args[0], "DOMAIN VIF CREDIT_IN_BYTES PERIOD_IN_USECS" + print "\nSet the credit limit of a virtual network interface." + + def main(self, args): + if len(args) != 5: self.err("%s: Invalid argument(s)" % args[0]) + dom = args[1] + v = map(int, args[2:5]) + server.xend_domain_vif_limit(dom, *v) + +xm.prog(ProgVifCreditLimit) + class ProgVifList(Prog): group = 'vif' name = 'vif-list' diff --git a/xen/include/public/io/domain_controller.h b/xen/include/public/io/domain_controller.h index c2abbc16be..491c309a0f 100644 --- a/xen/include/public/io/domain_controller.h +++ b/xen/include/public/io/domain_controller.h @@ -447,6 +447,7 @@ typedef struct { #define CMSG_NETIF_BE_DESTROY 1 /* Destroy a net-device interface. */ #define CMSG_NETIF_BE_CONNECT 2 /* Connect i/f to remote driver. */ #define CMSG_NETIF_BE_DISCONNECT 3 /* Disconnect i/f from remote driver. */ +#define CMSG_NETIF_BE_CREDITLIMIT 4 /* Limit i/f to a given credit limit. */ /* Messages to domain controller. */ #define CMSG_NETIF_BE_DRIVER_STATUS 32 @@ -510,6 +511,22 @@ typedef struct { u32 status; /* 8 */ } PACKED netif_be_destroy_t; /* 12 bytes */ +/* + * CMSG_NETIF_BE_CREDITLIMIT: + * Limit a virtual interface to "credit_bytes" bytes per "period_usec" + * microseconds. + */ +typedef struct { + /* IN */ + domid_t domid; /* 0: Domain attached to new interface. */ + u16 __pad0; /* 2 */ + u32 netif_handle; /* 4: Domain-specific interface handle. */ + u32 credit_bytes; /* 8: Vifs credit of bytes per period. */ + u32 period_usec; /* 12: Credit replenishment period. */ + /* OUT */ + u32 status; /* 16 */ +} PACKED netif_be_creditlimit_t; /* 20 bytes */ + /* * CMSG_NETIF_BE_CONNECT: * When the driver sends a successful response then the interface is fully -- 2.30.2